package com.chance.cc.crawler.development.scripts.kaola;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequestBody;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.core.tags.KVTag;
import com.chance.cc.crawler.core.tags.LabelTag;
import com.chance.cc.crawler.core.tags.crawler.CrawlerBusinessTags;
import com.chance.cc.crawler.development.scripts.allfeild.AICCommonField;
import com.chance.cc.crawler.development.scripts.allfeild.AutoVMCommonField;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.RandomUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.time.DateUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.FileInputStream;
import java.nio.charset.StandardCharsets;
import java.util.*;
import java.util.concurrent.LinkedBlockingQueue;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerDataType.*;

public class KaolaCrawlerScript extends CrawlerCommonScript {
    private Logger log = LoggerFactory.getLogger(KaolaCrawlerScript.class);//日志
    private static final String DOMAIN = "kaola";
    private static final String SITE = "commodity";
    private static final String REQUEST_AGAIN_TAG = DOMAIN + "_request_again";
    private static final Object obj = new Object();
    private Map head = new HashMap();


    //    https://search.kaola.com/
    private static final String HomeRegulars = "https://www.kaola.com";//主页
    //   https://m-search.kaola.com/goods/search.html?key=nike&zp=input&zn=h5Search
    //   https://m-search.kaola.com/api/searchlist.html

    //    https://search.kaola.com/search.html?key=nike&pageNo=3
    private static final String NewsFirstRegulars = "https://m-search.kaola.com/goods/search.html\\?key=\\S*&zp=input&zn=h5Search";//文章列表链接
    private static final String NewsNextRegulars = "https://m-search.kaola.com/api/searchlist.html";//文章列表链接

    //    https://g.kaola.com/__aggregation/goodsdetail/product?goodsId=9890450
    //    https://g.kaola.com/__aggregation/goodsdetail/product?goodsId=9462623

    //    https://goods.kaola.com/product/8837741.html
    private static final String ContentRegulars = "https://g.kaola.com/__aggregation/goodsdetail/product\\?goodsId=\\S*";//文章详情链接

    private static final String CommentRegulars = "https://goods.kaola.com/commentAjax/comment_list_for_wap.json";//评论下一页


    @Override
    public String domain() {
        return DOMAIN;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(HomeRegulars);
        addUrlRegular(NewsFirstRegulars);
        addUrlRegular(NewsNextRegulars);
        addUrlRegular(ContentRegulars);
        addUrlRegular(CommentRegulars);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        String site = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("site");
        return SITE.equals(site);
    }

    public List<CrawlerRecord> prepareRequest(CrawlerRequestRecord crawlerRequestRecord, List<CrawlerRequestRecord> supportSourceRecords) {
        List<CrawlerRecord> prepareLinks = new ArrayList<>();
        for (CrawlerRequestRecord supportSourceRecord : supportSourceRecords) {
            String url = supportSourceRecord.getHttpRequest().getUrl();
            if (url.contains("keys")) {
                this.initKeyWord(crawlerRequestRecord, supportSourceRecord);
            }
        }
        return prepareLinks;
    }

    //取出代表不同的车系key
    private void initKeyWord(CrawlerRequestRecord crawlerRequestRecord, CrawlerRequestRecord supportSourceRecord) {
        HttpPage internalDownloadPage = supportSourceRecord.getInternalDownloadPage();
        Json json = internalDownloadPage.getJson();
        String msg = json.jsonPath($_type + ".msg").get();
        if (!"success".equals(msg)) {
            log.error("keyword page [{}] download error!", internalDownloadPage.getRequest().getUrl());
            return;
        }
        List<String> keys = new ArrayList<>();
        List<String> all = json.jsonPath($_type + ".content").all();
        for (String data : all) {
            JSONObject jsonObject = JSONObject.parseObject(data);
            String keyword = jsonObject.getString("keyword");
            keys.add(keyword);
        }
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("keys", keys);

        LinkedBlockingQueue<String> uas = send2Redis("/data/chance_crawler_runner/logs/node/ua.txt");
//        LinkedBlockingQueue<String> uas = send2Redis("D:\\it\\idea\\chance-crawler-development\\crawler-dev-bootstrap\\src\\main\\java\\com\\chance\\cc\\crawler\\development\\bootstrap\\kaola\\ua.txt");
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV("uas", uas);

    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerRequestRecord> parsedLinks = new ArrayList();
        //        判断页面是否加载成功
        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        if (page.getStatusCode() != 200 || (!page.isDownloadSuccess())) {//||page.getRawText().contains("浏览器升级提示")
            log.error("download page url == {null} error status is {}", requestUrl, page.getStatusCode());
            if (page.getStatusCode() != 404) {//如果没有成功的下载  进行重新下载
                this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
                crawlerRecord.setNeedWashPage(false);
                return parsedLinks;
            }
        }

        if (crawlerRecord.getHttpRequest().getUrl().matches(HomeRegulars)) {
            homeRequest(crawlerRecord, page, parsedLinks);
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(NewsFirstRegulars) || crawlerRecord.getHttpRequest().getUrl().matches(NewsNextRegulars)) {
            newsRequest(crawlerRecord, page, parsedLinks);
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {
            articleRequest(crawlerRecord, page, parsedLinks);
        } else if (crawlerRecord.getHttpRequest().getUrl().matches(CommentRegulars)) {
            commentRequest(crawlerRecord, page, parsedLinks);
        }
        return parsedLinks;
    }

    //    解析动态
    private void homeRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        KVTag keys = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("keys");
        List<String> name = (List<String>) keys.getVal();
        for (int i = 0; i < name.size(); i++) {//name.size()
//            String url = "https://search.kaola.com/search.html?key=" + name.get(i) + "&pageNo=1";
            crawlerRecord.tagsCreator().bizTags().addCustomKV("key", name.get(i));
            crawlerRecord.tagsCreator().bizTags().addCustomKV("pageNo", "1");
            String url = "https://m-search.kaola.com/goods/search.html?key=" + name.get(i) + "&zp=input&zn=h5Search";
            CrawlerRequestRecord record = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(url)
                    .releaseTime(Long.valueOf(System.currentTimeMillis()))
                    .copyBizTags()
                    .copyResultTags()
                    .build();
            parsedLinks.add(record);
        }
    }

    private void newsRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String url = page.getRequest().getUrl();
        try {
            if (url.matches(NewsFirstRegulars) || url.matches(NewsNextRegulars)) {
                synchronized (obj) {
                    int v = (int) new Random().nextDouble() * 3000;
                    Thread.sleep(4000 + v);
                    String key = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("key");
                    String pageNo = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("pageNo");
                    if (pageNo == null) {
                        String[] split = url.replaceAll("key=", "/").replaceAll("&zp", "/").split("/");
                        key = split[split.length - 2];
                        pageNo = "1";
                    }
                    pageNo = Integer.parseInt(pageNo) + 1 + "";
                    crawlerRecord.tagsCreator().bizTags().addCustomKV("pageNo", pageNo);
                    crawlerRecord.tagsCreator().bizTags().addCustomKV("key", key);
                    String json = "{\"pageSize\":20,\"pageNo\":" + pageNo + ",\"searchType\":\"synonym\",\"sortType\":0,\"desc\":1,\"key\":\"" + key + "\",\"hasMore\":true,\"filterList\":[{\"hidden\":false,\"itemId\":11,\"itemName\":\"考拉服务\",\"itemType\":0,\"list\":[],\"lower\":-1,\"showLine\":0,\"upper\":-1},{\"hidden\":false,\"itemId\":12,\"itemName\":\"价格\",\"itemType\":1,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"itemId\":13,\"itemName\":\"品牌\",\"itemType\":2,\"list\":[],\"lower\":-1,\"maxShowLine\":8,\"showLine\":3,\"upper\":-1},{\"hidden\":false,\"itemId\":14,\"itemName\":\"运动用品\",\"itemType\":0,\"list\":[],\"lower\":-1,\"showLine\":2,\"upper\":-1},{\"hidden\":false,\"itemId\":14,\"itemName\":\"母婴\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100251\",\"itemId\":15,\"itemName\":\"尺码\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":7,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100224\",\"itemId\":15,\"itemName\":\"适用群体\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":2,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100410\",\"itemId\":15,\"itemName\":\"功能\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":4,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100234\",\"itemId\":15,\"itemName\":\"适用场景\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":4,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100304\",\"itemId\":15,\"itemName\":\"鞋帮高度\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"501703\",\"itemId\":15,\"itemName\":\"运动鞋科技\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100280\",\"itemId\":15,\"itemName\":\"适用年龄\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"itemId\":16,\"itemName\":\"国家/地区\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":2,\"showLine\":1,\"upper\":-1}],\"isMarket\":false,\"marketCategory\":\"\",\"storeCount\":0,\"noStoreCount\":0,\"t\":1634266230383}";
                    String nextPageUrl = "https://m-search.kaola.com/api/searchlist.html";
                    CrawlerRequestRecord turnPageRequest = CrawlerRequestRecord.builder()
                            .itemPageRequest(crawlerRecord)
                            .recordKey(nextPageUrl)
                            .httpUrl(nextPageUrl)
                            .releaseTime(System.currentTimeMillis())
                            .copyBizTags()
                            .copyResultTags()
                            .needParsed(true)
                            .httpHead("authority", "m-search.kaola.com")
                            .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                            .httpHead("x-requested-with", "XMLHttpRequest")
                            .httpHead("sec-ch-ua-mobile", "?1")
                            .httpHead("user-agent", uas(crawlerRecord))
                            .httpHead("content-type", "application/json")
                            .httpHead("accept", "application/json")
                            .httpHead("origin", "https://goods.kaola.com")
                            .httpHead("sec-fetch-site", "same-origin")
                            .httpHead("sec-fetch-mode", "cors")
                            .httpHead("sec-fetch-dest", "empty")
//                            .httpHead("referer", page.getRequest().getUrl())
                            .httpHead("accept-language", "zh-CN,zh;q=0.9")
//                            .httpHead("cookie", "kaola_user_key=f056248c-62ea-4836-9e89-0f88abbe95ca; _klhtxd_=31; cna=pYo7Gb9ywFsCAXWfEdrbhWHe; davisit=1; __da_ntes_utma=2525167.538674451.1634029726.1634029726.1634029726.1; __da_ntes_utmz=2525167.1634029726.1.1.; _samesite_flag_=true; t=6f43daa980d6e5e2c68184992a3b0dd2; _tb_token_=e6e3b38b45e7a; __da=__da_dad3e203_5c4bd87a7ff92c00; gtm_dsp_tag=__da_dad3e203_5c4bd87a7ff92c00; __da_ntes_utmfc=utmcsr%3Dwww.kaola.com%7Cutmccn%3D(prom)%7Cutmcmd%3Dprom%7Cutmpclid%3D__da_dad3e203_5c4bd87a7ff92c00; KAOLA_NEW_USER_COOKIE=yes; ucn=center; _uab_collina=163403191133936507985119; JSESSIONID-WKL-8IO=cBgixz%5CYkIOiJ5c0d5meXjoJ%2BvaNZY0uMwp1Zeh63svHqYcMABybcxV%5CI2bnYADCCk%2BuLTARzhBQGCL%5CUP09%2FL0CYOG2cOwGaetdpW%2F3cuRxmsRvAAafG5zmHN1ydiarh%2Bvp%5CjPvnW1Z4te4v9W3e%2Bwk73WQjJ61esIsYDY7%5CplTIvkW%3A1634198898117; xlly_s=1; KAOLA_USER_ID.sig=wQJGcXAMGbLxoAFUjk4TinI3LkO8PHtzoCmuTm3jnF4; NTES_KAOLA_RV=9985701|8962272|10319343|9637425|9935592|10640661|10575166|9749434|10330839|8245655|9462623|9900584|8825110|8837741|9739796|10453662|8740110; cookie2=19d713f27b8d9068360cd01503c209b2; unionID=z_uq_gb9xxuf2zhuanke1634190330224; __da_ntes_utmb=2525167.1.10.1634190333; hb_MA-AE38-1FCC6CD7201B_source=account.kaola.com; csg=3c8c988a; NTES_OSESS=fad6be702f1e47f88b3a252e4a66a55e; KAOLA_USER_ID=109999078954674946; KAOLA_MAIN_ACCOUNT=163403154599421365@pvkaola.163.com; unb=2212772097084; kaola_csg=c3d81ea9; kaola-user-beta-traffic=14618303749; firstLogin=0; x5sec=7b227761676272696467652d616c69626162612d67726f75703b32223a2233623365613739666639393132646462393564343531353332353132366566384350794a6e3473474549763279744f6438346d5579774561447a49794d5449334e7a49774f5463774f4451374d5367434d49503437616e392f2f2f2f2f77453d227d; isg=BMrKodEin2K5aBPM1PKDip1pG7Bsu04VHzsgnVQDdp2oB2rBPEueJRD1E3Pb7Mat; tfstk=cUvFBnMdlvHFQ8BSM96rFXT5-NtdwWQllRSVtBX2Uu3kZMfmbzIMaRjN4ZHGt; l=eB_prBgmgJ12oM9zBOfanurza77OSIRYYuPzaNbMiOCPOWfB5ayCW6ElY586C3GVh6yWR3oq4UgkBeYBq7VonxvtDGO5MsHmn")
                            .build();
                    turnPageRequest.getHttpRequest().setMethod("POST");
                    turnPageRequest.getHttpRequest().setRequestBody(HttpRequestBody.json(json, "UTF-8"));
                    turnPageRequest.getHttpConfig().setHttpSite("search");//区分site
                    turnPageRequest.getHttpConfig().setDisableCookie(false);//禁用cookies
                    parsedLinks.add(turnPageRequest);
                }
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }

        List<Selectable> nodes = null;
        if (url.matches(NewsFirstRegulars)) {
            String[] split = page.getRawText().split("var firstSearchResult = window.firstSearchResult =");
            if (split.length > 1) {
                String[] jsons = split[1].split(";</script> ");
                nodes = new Json(jsons[0]).jsonPath($_type + "..goodsList[*]").nodes();
            }
        } else {
            nodes = page.getJson().jsonPath($_type + "..goodsList[*]").nodes();
        }
        for (Selectable node : nodes) {
            String urlId = new Json(node.get()).jsonPath(".goodsId").get();
            if (StringUtils.isBlank(urlId)) {
                continue;
            }
            String articleUrl = "https://g.kaola.com/__aggregation/goodsdetail/product?goodsId=" + urlId;
            CrawlerRequestRecord articleRecord = CrawlerRequestRecord.builder()//解析文章
                    .itemPageRequest(crawlerRecord)
                    .recordKey(articleUrl)
                    .httpUrl(articleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .needWashed(true)
                    .build();
            parsedLinks.add(articleRecord);
        }
    }

    private void articleRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String[] split1 = page.getRawText().split("__INITIAL_DATA__=");
        if (split1.length < 2) {
            this.requestAgainCrawlerRecord(parsedLinks, crawlerRecord);
            crawlerRecord.setNeedWashPage(false);
        } else {
            crawlerRecord.tagsCreator().bizTags().addCustomKV(AICCommonField.Field_Urls, page.getRequest().getUrl());
            String[] split = page.getRequest().getUrl().replaceAll("goodsId=", "/").split("/");
            String goodsId = split[split.length - 1];
            String paginationContext = "null";
            String pageNo = "1";

            Map<String, Object> map = new HashMap<>();
            map.put("goodsId", goodsId);
            map.put("pageNo", pageNo);
            map.put("pageSize", "10");
            map.put("tagType", "0");
            map.put("tagName", "全部");
            map.put("paginationContext", "[object Object]");
            map.put("showSelfGoodsComment", "false");

            crawlerRecord.tagsCreator().bizTags().addCustomKV("pageNoComment", pageNo);
            String nextPageUrl = "https://goods.kaola.com/commentAjax/comment_list_for_wap.json";
            KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
            CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
            CrawlerRequestRecord articleRecord = CrawlerRequestRecord.builder()//解析文章
                    .itemPageRequest(crawlerRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .needWashed(true)
                    .notFilterRecord()
                    .httpHead("authority", "goods.kaola.com")
                    .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                    .httpHead("accept", "application/json, text/plain, */*")
                    .httpHead("sec-ch-ua-mobile", "?1")
                    .httpHead("user-agent", uas(crawlerRecord))
                    .httpHead("content-type", "application/x-www-form-urlencoded")
                    .httpHead("origin", "https://goods.kaola.com")
                    .httpHead("sec-fetch-site", "same-origin")
                    .httpHead("sec-fetch-mode", "cors")
                    .httpHead("sec-fetch-dest", "empty")
                    .httpHead("referer", "https://g.kaola.com/")
                    .httpHead("accept-language", "zh-CN,zh;q=0.9")
//                    .httpHead("cookie", "kaola_user_key=f056248c-62ea-4836-9e89-0f88abbe95ca; _klhtxd_=31; cna=pYo7Gb9ywFsCAXWfEdrbhWHe; xlly_s=1; __da_ntes_utma=2525167.538674451.1634029726.1634029726.1634029726.1; davisit=1; __da_ntes_utmz=2525167.1634029726.1.1.; _samesite_flag_=true; cookie2=11e22af35f81726bd0824cfd2587069b; t=6f43daa980d6e5e2c68184992a3b0dd2; _tb_token_=e6e3b38b45e7a; __da=__da_dad3e203_5c4bd87a7ff92c00; gtm_dsp_tag=__da_dad3e203_5c4bd87a7ff92c00; __da_ntes_utmfc=utmcsr%3Dwww.kaola.com%7Cutmccn%3D(prom)%7Cutmcmd%3Dprom%7Cutmpclid%3D__da_dad3e203_5c4bd87a7ff92c00; KAOLA_NEW_USER_COOKIE=yes; csg=0cfb4691; NTES_OSESS=cacf12dfff83458e82cb0316e1e91e02; KAOLA_USER_ID=109999078954674946; KAOLA_MAIN_ACCOUNT=163403154599421365@pvkaola.163.com; unb=2212772097084; kaola_csg=77844cd9; kaola-user-beta-traffic=14618303749; ucn=center; KAOLA_USER_ID.sig=wQJGcXAMGbLxoAFUjk4TinI3LkO8PHtzoCmuTm3jnF4; _uab_collina=163403191133936507985119; __da_ntes_utmb=2525167.1.10.1634112444; JSESSIONID-WKL-8IO=cBgixz%5CYkIOiJ5c0d5meXjoJ%2BvaNZY0uMwp1Zeh63svHqYcMABybcxV%5CI2bnYADCCk%2BuLTARzhBQGCL%5CUP09%2FL0CYOG2cOwGaetdpW%2F3cuRxmsRvAAafG5zmHN1ydiarh%2Bvp%5CjPvnW1Z4te4v9W3e%2Bwk73WQjJ61esIsYDY7%5CplTIvkW%3A1634198898117; NTES_KAOLA_RV=10319343|9637425|9935592|10640661|10575166|9749434|10330839|8245655|9462623|9900584|8825110|8837741|9739796|10453662|8740110; unionID=z_uq_5pa6jsufzhuanke1634114262529; hb_MA-AE38-1FCC6CD7201B_source=c.duomai.com; x5sec=7b227761676272696467652d616c69626162612d67726f75703b32223a223565333939636332636430626666633864323463626662333765653530643065434943376d6f7347454f477638502b673970662f31774561447a49794d5449334e7a49774f5463774f4451374d6a43442b4f32702f662f2f2f2f3842227d; isg=BDk51OvHfE4adCDZm58AlwqMSKUTRi34SD6zqFtutWDf4ll0o5Y9yKewYOYUwcUw; l=eB_prBgmgJ12oY7DBOfanurza77OSIRYYuPzaNbMiOCPO0fB5TPdW6EySxY6C3GVh6yWR3oq4UgkBeYBq7VonxvtDGO5MsHmn; tfstk=c2eNBn6tPOBaZ1MfwAMVhE9KZbZOwP0mPpors5DDfo_0-cfcYEn3gpmE04Bns")
                    .build();
            articleRecord.setFilter(filterInfoRecord.getFilter());
            articleRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
            articleRecord.getHttpRequest().setMethod("POST");
            articleRecord.getHttpRequest().setRequestBody(HttpRequestBody.form(map, "UTF-8"));
            articleRecord.getHttpConfig().setHttpSite("search");//区分site
            articleRecord.getHttpConfig().setDisableCookie(false);//禁用cookies
            parsedLinks.add(articleRecord);
        }
    }

    private void commentRequest(CrawlerRequestRecord crawlerRecord, HttpPage page, List<CrawlerRequestRecord> parsedLinks) {
        String commnets = page.getJson().jsonPath($_type + "..totalCount").get();
        if (commnets == null) {
            commnets = "0";
        }
        String articleUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);
        String pageNo = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("pageNoComment");
        try {

            String[] split = articleUrl.replaceAll("goodsId=", "/").split("/");
            String goodsId = split[split.length - 1];
            int pageNoNext = Integer.parseInt(pageNo) + 1;

            Map<String, Object> map = new HashMap<>();
            map.put("goodsId", goodsId);
            map.put("pageNo", pageNoNext + "");
            map.put("pageSize", "10");
            map.put("tagType", "0");
            map.put("tagName", "全部");
            map.put("paginationContext", "[object Object]");
            map.put("showSelfGoodsComment", "false");

            crawlerRecord.tagsCreator().bizTags().addCustomKV("pageNoComment", pageNoNext + "");
            if (Integer.parseInt(commnets) - Integer.parseInt(pageNo) * 10 > 0) {
                String nextPageUrl = "https://goods.kaola.com/commentAjax/comment_list_for_wap.json";
                KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
                CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
                CrawlerRequestRecord commentRecord = CrawlerRequestRecord.builder()//解析文章
                        .itemPageRequest(crawlerRecord)
                        .recordKey(nextPageUrl)
                        .httpUrl(nextPageUrl)
                        .releaseTime(System.currentTimeMillis())
                        .copyBizTags()
                        .copyResultTags()
                        .needWashed(true)
                        .notFilterRecord()
                        .httpHead("authority", "goods.kaola.com")
                        .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                        .httpHead("accept", "application/json, text/plain, */*")
                        .httpHead("sec-ch-ua-mobile", "?1")
                        .httpHead("user-agent", uas(crawlerRecord))
                        .httpHead("content-type", "application/x-www-form-urlencoded")
                        .httpHead("origin", "https://goods.kaola.com")
                        .httpHead("sec-fetch-site", "same-origin")
                        .httpHead("sec-fetch-mode", "cors")
                        .httpHead("sec-fetch-dest", "empty")
                        .httpHead("referer", "https://g.kaola.com/")
                        .httpHead("accept-language", "zh-CN,zh;q=0.9")
//                    .httpHead("cookie", "kaola_user_key=f056248c-62ea-4836-9e89-0f88abbe95ca; _klhtxd_=31; cna=pYo7Gb9ywFsCAXWfEdrbhWHe; xlly_s=1; __da_ntes_utma=2525167.538674451.1634029726.1634029726.1634029726.1; davisit=1; __da_ntes_utmz=2525167.1634029726.1.1.; _samesite_flag_=true; cookie2=11e22af35f81726bd0824cfd2587069b; t=6f43daa980d6e5e2c68184992a3b0dd2; _tb_token_=e6e3b38b45e7a; __da=__da_dad3e203_5c4bd87a7ff92c00; gtm_dsp_tag=__da_dad3e203_5c4bd87a7ff92c00; __da_ntes_utmfc=utmcsr%3Dwww.kaola.com%7Cutmccn%3D(prom)%7Cutmcmd%3Dprom%7Cutmpclid%3D__da_dad3e203_5c4bd87a7ff92c00; KAOLA_NEW_USER_COOKIE=yes; csg=0cfb4691; NTES_OSESS=cacf12dfff83458e82cb0316e1e91e02; KAOLA_USER_ID=109999078954674946; KAOLA_MAIN_ACCOUNT=163403154599421365@pvkaola.163.com; unb=2212772097084; kaola_csg=77844cd9; kaola-user-beta-traffic=14618303749; ucn=center; KAOLA_USER_ID.sig=wQJGcXAMGbLxoAFUjk4TinI3LkO8PHtzoCmuTm3jnF4; _uab_collina=163403191133936507985119; __da_ntes_utmb=2525167.1.10.1634112444; JSESSIONID-WKL-8IO=cBgixz%5CYkIOiJ5c0d5meXjoJ%2BvaNZY0uMwp1Zeh63svHqYcMABybcxV%5CI2bnYADCCk%2BuLTARzhBQGCL%5CUP09%2FL0CYOG2cOwGaetdpW%2F3cuRxmsRvAAafG5zmHN1ydiarh%2Bvp%5CjPvnW1Z4te4v9W3e%2Bwk73WQjJ61esIsYDY7%5CplTIvkW%3A1634198898117; NTES_KAOLA_RV=10319343|9637425|9935592|10640661|10575166|9749434|10330839|8245655|9462623|9900584|8825110|8837741|9739796|10453662|8740110; unionID=z_uq_5pa6jsufzhuanke1634114262529; hb_MA-AE38-1FCC6CD7201B_source=c.duomai.com; x5sec=7b227761676272696467652d616c69626162612d67726f75703b32223a223565333939636332636430626666633864323463626662333765653530643065434943376d6f7347454f477638502b673970662f31774561447a49794d5449334e7a49774f5463774f4451374d6a43442b4f32702f662f2f2f2f3842227d; isg=BDk51OvHfE4adCDZm58AlwqMSKUTRi34SD6zqFtutWDf4ll0o5Y9yKewYOYUwcUw; l=eB_prBgmgJ12oY7DBOfanurza77OSIRYYuPzaNbMiOCPO0fB5TPdW6EySxY6C3GVh6yWR3oq4UgkBeYBq7VonxvtDGO5MsHmn; tfstk=c2eNBn6tPOBaZ1MfwAMVhE9KZbZOwP0mPpors5DDfo_0-cfcYEn3gpmE04Bns")
                        .build();
                commentRecord.setFilter(filterInfoRecord.getFilter());
                commentRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
                commentRecord.getHttpRequest().setMethod("POST");
                commentRecord.getHttpRequest().setRequestBody(HttpRequestBody.form(map, "UTF-8"));
                commentRecord.getHttpConfig().setHttpSite("search");//区分site
                commentRecord.getHttpConfig().setDisableCookie(false);//禁用cookies
                parsedLinks.add(commentRecord);
            }
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
    }


    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerDataList = new ArrayList<>();
        crawlerRecord.tagsCreator().bizTags().addCustomKV("uas", null);
        if (crawlerRecord.getHttpRequest().getUrl().matches(ContentRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(articleListWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(article)) {
                crawlerDataList.addAll(articleWash(crawlerRecord, page));
            }
        }
        if (crawlerRecord.getHttpRequest().getUrl().matches(CommentRegulars)) {
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(comment)) {
                crawlerDataList.addAll(commentWash(crawlerRecord, page));
            }
            if (crawlerRecord.tagsCreator().resultTags().hasDataType(interaction)) {
                crawlerDataList.addAll(interactionWash(crawlerRecord, page));
            }
        }

        return crawlerDataList;
    }

    //清洗文章链接
    public List<CrawlerData> articleListWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("清洗文章链接");
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            String articleUrl = page.getRequest().getUrl();
            String[] articleId = articleUrl.replaceAll("goodsld=", "/").split("/");
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, article.enumVal(), articleId[articleId.length - 1]))
                    .resultLabelTag(article)
                    .url(articleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV("releaseTimeToLong", System.currentTimeMillis() + "")
                    .addContentKV("articleUrl", articleUrl)
                    .addContentKV(AICCommonField.Tag_Site_Info, "商品")
                    .flowInPipelineTag("redis")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }

    //解析文章
    public List<CrawlerData> articleWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        log.info("解析文章");
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        try {
            String[] split1 = page.getRawText().split("__INITIAL_DATA__=");
            String json = split1[1].split("</script>")[0];

            String articleUrl = page.getRequest().getUrl();//文章url
            String title = new Json(json).jsonPath($_type + "..goodsStatic.title").get();//标题
            String price = new Json(json).jsonPath($_type + "..price").get();//价格
            List<String> picture = new Json(json).jsonPath($_type + "..bannerImgUrlList[*]").all();//图片
            StringBuffer pictures = new StringBuffer();//拼接图片
            for (String text : picture) {
                pictures.append(text).append("\0x1");
            }
            String[] split = articleUrl.replaceAll("goodsId=", "/").trim().split("/");//关键字
            String articleId = split[split.length - 1];
            CrawlerData crawlerData = CrawlerData.builder()
                    .data(crawlerRecord, page)
                    .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, article.enumVal(), articleId))
                    .resultLabelTag(article)
                    .url(articleUrl)
                    .releaseTime(System.currentTimeMillis())
                    .addContentKV(AICCommonField.Field_Title, title)
                    .addContentKV(AICCommonField.Field_Produce_Price, price)
                    .addContentKV(AICCommonField.Field_Images, pictures.toString())
                    .flowInPipelineTag("kafka")
                    .build();
            crawlerData.setFilterPipelineResult(true);
            crawlerArticleDataList.add(crawlerData);
        } catch (Exception e) {
            log.error(e.getMessage(), e);
        }
        return crawlerArticleDataList;
    }

    //  评论清洗
    private Collection<? extends CrawlerData> commentWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Selectable> nodes = page.getJson().jsonPath($_type + "..list[*]").nodes();
        for (Selectable node : nodes) {
            log.info("评论清洗");
            try {
                String content = new Json(node.get()).jsonPath(".commentContent").get();//评论内容
                String releaseTime = new Json(node.get()).jsonPath(".createTime").get();//回复时间
                String commentId = new Json(node.get()).jsonPath(".goodsCommentId").get();//评论id

                List<String> picture = new Json(node.get()).jsonPath(".imgUrls[*]").all();//图片
                StringBuffer pictures = new StringBuffer();//拼接图片
                for (String text : picture) {
                    pictures.append(text).append("\0x1");
                }

                String articleUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url
                String[] split = articleUrl.replaceAll("goodsId=", "/").split("/");//关键字
                String articleId = split[split.length - 1];
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, comment.enumVal(), commentId))
                        .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, article.enumVal(), articleId))
                        .resultLabelTag(comment)
                        .url(articleUrl)
                        .releaseTime(Long.parseLong(releaseTime))
                        .addContentKV(AICCommonField.Field_Content, content)
                        .addContentKV(AICCommonField.Field_Images, pictures.toString().trim())
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    //  评论互动量清洗
    private Collection<? extends CrawlerData> interactionWash(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        List<CrawlerData> crawlerArticleDataList = new ArrayList<>();
        List<Selectable> nodes = page.getJson().jsonPath($_type + "..list[*]").nodes();
        for (Selectable node : nodes) {
            log.info("评论互动量清洗");
            try {
                String likes = new Json(node.get()).jsonPath(".zanCount").get();//点赞
                String releaseTime = new Json(node.get()).jsonPath(".createTime").get();//回复时间
                String commentId = new Json(node.get()).jsonPath(".goodsCommentId").get();//评论id

                String articleUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);//文章url
                CrawlerData crawlerData = CrawlerData.builder()
                        .data(crawlerRecord, page)
                        .dataId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, interaction.enumVal(), commentId))
                        .parentId(StringUtils.joinWith("-", crawlerRecord.getDomain(), SITE, comment.enumVal(), commentId))
                        .resultLabelTag(interaction)
                        .url(articleUrl)
                        .releaseTime(Long.parseLong(releaseTime))
                        .addContentKV(AICCommonField.Field_I_Likes, likes)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.result)
                        .requestLabelTag(CrawlerEnum.CrawlerRequestType.filter)
                        .flowInPipelineTag("kafka")
                        .build();
                crawlerData.setFilterPipelineResult(true);
                crawlerArticleDataList.add(crawlerData);
            } catch (Exception e) {
                log.error(e.getMessage(), e);
            }
        }
        return crawlerArticleDataList;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {
    }

    /*
     * 重新下载
     * */
    private void requestAgainCrawlerRecord(List<CrawlerRequestRecord> crawlerRequestRecords, CrawlerRequestRecord crawlerRecord) {
        CrawlerBusinessTags crawlerBusinessTags = crawlerRecord.tagsCreator().bizTags();
        int count = 0;
        if (crawlerBusinessTags.hasKVTag(REQUEST_AGAIN_TAG)) {
            count = (int) crawlerBusinessTags.getCategoryTag().getKVTag(REQUEST_AGAIN_TAG).getVal();
            if (count >= 50) {
                log.error("autohome download page the number of retries exceeds the limit" +
                        ",request url {}", crawlerRecord.getHttpRequest().getUrl());
                return;
            }
        }

        String requestUrl = crawlerRecord.getHttpRequest().getUrl();
        CrawlerRequestRecord crawlerRequestRecord = null;
        LabelTag type = crawlerRecord.tagsCreator().requestTags().getCategoryTag().getLabelTags().get("turn_page_item_request");
        if (type == null) {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .turnPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey())//+ count
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .notFilterRecord()
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        } else if (requestUrl.matches(NewsNextRegulars)) {
            String key = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("key");
            String pageNo = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("pageNo");
            crawlerRecord.tagsCreator().bizTags().addCustomKV("pageNo", pageNo);
            crawlerRecord.tagsCreator().bizTags().addCustomKV("key", key);
            String json = "{\"pageSize\":20,\"pageNo\":" + pageNo + ",\"searchType\":\"synonym\",\"sortType\":0,\"desc\":1,\"key\":\"" + key + "\",\"hasMore\":true,\"filterList\":[{\"hidden\":false,\"itemId\":11,\"itemName\":\"考拉服务\",\"itemType\":0,\"list\":[],\"lower\":-1,\"showLine\":0,\"upper\":-1},{\"hidden\":false,\"itemId\":12,\"itemName\":\"价格\",\"itemType\":1,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"itemId\":13,\"itemName\":\"品牌\",\"itemType\":2,\"list\":[],\"lower\":-1,\"maxShowLine\":8,\"showLine\":3,\"upper\":-1},{\"hidden\":false,\"itemId\":14,\"itemName\":\"运动用品\",\"itemType\":0,\"list\":[],\"lower\":-1,\"showLine\":2,\"upper\":-1},{\"hidden\":false,\"itemId\":14,\"itemName\":\"母婴\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100251\",\"itemId\":15,\"itemName\":\"尺码\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":7,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100224\",\"itemId\":15,\"itemName\":\"适用群体\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":2,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100410\",\"itemId\":15,\"itemName\":\"功能\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":4,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100234\",\"itemId\":15,\"itemName\":\"适用场景\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":4,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100304\",\"itemId\":15,\"itemName\":\"鞋帮高度\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"501703\",\"itemId\":15,\"itemName\":\"运动鞋科技\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"id\":\"100280\",\"itemId\":15,\"itemName\":\"适用年龄\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":1,\"showLine\":1,\"upper\":-1},{\"hidden\":false,\"itemId\":16,\"itemName\":\"国家/地区\",\"itemType\":0,\"list\":[],\"lower\":-1,\"maxShowLine\":2,\"showLine\":1,\"upper\":-1}],\"isMarket\":false,\"marketCategory\":\"\",\"storeCount\":0,\"noStoreCount\":0,\"t\":1634266230383}";
            String nextPageUrl = "https://m-search.kaola.com/api/searchlist.html";
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .needParsed(true)
                    .httpHead("authority", "m-search.kaola.com")
                    .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                    .httpHead("x-requested-with", "XMLHttpRequest")
                    .httpHead("sec-ch-ua-mobile", "?1")
                    .httpHead("user-agent", uas(crawlerRecord))
                    .httpHead("content-type", "application/json")
                    .httpHead("accept", "application/json")
                    .httpHead("origin", "https://goods.kaola.com")
                    .httpHead("sec-fetch-site", "same-origin")
                    .httpHead("sec-fetch-mode", "cors")
                    .httpHead("sec-fetch-dest", "empty")
                    .httpHead("accept-language", "zh-CN,zh;q=0.9")
                    .build();
            crawlerRequestRecord.getHttpRequest().setMethod("POST");
            crawlerRequestRecord.getHttpRequest().setRequestBody(HttpRequestBody.json(json, "UTF-8"));
            crawlerRequestRecord.getHttpConfig().setHttpSite("search");//区分site
            crawlerRequestRecord.getHttpConfig().setDisableCookie(false);//禁用cookies
        } else if (requestUrl.matches(CommentRegulars)) {
            String articleUrl = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal(AICCommonField.Field_Urls);
            String pageNo = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTagStrVal("pageNoComment");

            String[] split = articleUrl.replaceAll("goodsId=", "/").split("/");
            String goodsId = split[split.length - 1];

            Map<String, Object> map = new HashMap<>();
            map.put("goodsId", goodsId);
            map.put("pageNo", pageNo);
            map.put("pageSize", "10");
            map.put("tagType", "0");
            map.put("tagName", "全部");
            map.put("paginationContext", "[object Object]");
            map.put("showSelfGoodsComment", "false");

            crawlerRecord.tagsCreator().bizTags().addCustomKV("pageNoComment", pageNo);
            String nextPageUrl = "https://goods.kaola.com/commentAjax/comment_list_for_wap.json";
            KVTag filterInfoTag = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("comment_record_filter_info");
            CrawlerRecord filterInfoRecord = JSON.parseObject((String) filterInfoTag.getVal(), CrawlerRecord.class);
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .recordKey(nextPageUrl)
                    .httpUrl(nextPageUrl)
                    .releaseTime(System.currentTimeMillis())
                    .copyBizTags()
                    .copyResultTags()
                    .needWashed(true)
                    .notFilterRecord()
                    .httpHead("authority", "goods.kaola.com")
                    .httpHead("sec-ch-ua", "\"Google Chrome\";v=\"93\", \" Not;A Brand\";v=\"99\", \"Chromium\";v=\"93\"")
                    .httpHead("accept", "application/json, text/plain, */*")
                    .httpHead("sec-ch-ua-mobile", "?1")
                    .httpHead("user-agent", uas(crawlerRecord))
                    .httpHead("content-type", "application/x-www-form-urlencoded")
                    .httpHead("origin", "https://goods.kaola.com")
                    .httpHead("sec-fetch-site", "same-origin")
                    .httpHead("sec-fetch-mode", "cors")
                    .httpHead("sec-fetch-dest", "empty")
                    .httpHead("referer", "https://g.kaola.com/")
                    .httpHead("accept-language", "zh-CN,zh;q=0.9")
                    .build();
            crawlerRequestRecord.setFilter(filterInfoRecord.getFilter());
            crawlerRequestRecord.setFilterInfos(filterInfoRecord.getFilterInfos());
            crawlerRequestRecord.getHttpRequest().setMethod("POST");
            crawlerRequestRecord.getHttpRequest().setRequestBody(HttpRequestBody.form(map, "UTF-8"));
            crawlerRequestRecord.getHttpConfig().setHttpSite("search");//区分site
            crawlerRequestRecord.getHttpConfig().setDisableCookie(false);//禁用cookies
        } else {
            crawlerRequestRecord = CrawlerRequestRecord.builder()
                    .itemPageRequest(crawlerRecord)
                    .httpUrl(requestUrl)
                    .recordKey(crawlerRecord.getRecordKey())//+ count
                    .releaseTime(crawlerRecord.getReleaseTime())
                    .copyBizTags()
                    .copyResultTags()
                    .build();
        }

        crawlerRequestRecord.setNeedWashPage(crawlerRecord.isNeedWashPage());
        crawlerRequestRecord.setNeedParsedPage(crawlerRecord.isNeedParsedPage());
        crawlerRequestRecord.getHttpRequest().setHeaders(crawlerRecord.getHttpRequest().getHeaders());
        crawlerRequestRecord.getHttpRequest().setExtras(crawlerRecord.getHttpRequest().getExtras());
        crawlerRequestRecord.tagsCreator().bizTags().addCustomKV(REQUEST_AGAIN_TAG, ++count);
        crawlerRequestRecords.add(crawlerRequestRecord);
    }

    private static String uas(CrawlerRequestRecord crawlerRecord) {
        KVTag val = crawlerRecord.tagsCreator().bizTags().getCategoryTag().getKVTag("uas");
        LinkedBlockingQueue<String> uas = (LinkedBlockingQueue<String>) val.getVal();
        String poll = uas.poll();
        uas.add(poll);
        return poll;
    }

    //    添加ua
    public static LinkedBlockingQueue<String> send2Redis(String filePath) {
        List<String> uasStr = null;
        try {
            uasStr = IOUtils.readLines(new FileInputStream(filePath), StandardCharsets.UTF_8);
        } catch (Exception e) {
            System.out.println("*----------------*");
            e.printStackTrace();
        }
        if (uasStr != null) {
            LinkedBlockingQueue<String> uas = new LinkedBlockingQueue<>(200000);
            uas.addAll(uasStr);
            return uas;
        } else {
            return null;
        }
    }

}
